package dianshiju

import downloader.FilePipeline
import downloader.biquge.BiqugePageProcessor
import us.codecraft.webmagic.Spider
import us.codecraft.webmagic.pipeline.ConsolePipeline

class Parser {
}

fun main(args: Array<String>) {
    val sp = Spider.create(YoukuPageProcessor())
            .addUrl("http://list.youku.com/category/show/c_97.html",
                    "http://list.youku.com/category/show/c_85.html")
            .addPipeline(dianshiju.FilePipeline())
            .thread(8)
    sp.run()




}